import json
from datetime import datetime

import scrapy
from spidertools.common_pipeline.base_item import convert_dict
from spidertools.utils.time_utils import get_current_date

from commonresources.spider_items.jiangsu.zhenjiang.items import ZhenJiangGongGongZiYuanJiaoYiZhongXinItem


class ZhenJiangGongGongZiYuanJiaoYiZhongXinSpider(scrapy.Spider):
    """
        镇江公共资源交易中心     http://ggzy.zhenjiang.gov.cn/
    """
    name = 'ZhenJiangGongGongZiYuanJiaoYiZhongXin'
    name_zh = "镇江公共资源交易中心"
    province = "江苏"
    city = '镇江'
    allowed_domains = ['ggzy.zhenjiang.gov.cn']

    # start_urls = ['http://ggzy.zhenjiang.gov.cn/']

    def __init__(self, full_dose=False):
        '''        :param full_dose: 是否全量爬取，默认为false
        '''
        self.browser_cookie = {}
        self.convert_dict = convert_dict
        self.full_dose = full_dose
        super().__init__()

    def parse(self, response):
        pass

    def classify1(self):
        classify = {  # test_pdf
            "jsgc": [
                {"construction_type": '建设工程'},
                {
                    "categorynum": "001001002",
                    "categoryname": "招标公告"
                },

            ],

        }
        return classify

    def classify(self):
        classify = {
            "jsgc": [
                {"construction_type": '建设工程'},
                {
                    "categorynum": "001001008",
                    "categoryname": "标前预公示"
                },
                {
                    "categorynum": "001001002",
                    "categoryname": "招标公告"
                },
                {
                    "categorynum": "001001003",
                    "categoryname": "未入围公示"
                },
                {
                    "categorynum": "001001004",
                    "categoryname": "评标结果公示"
                },
                {
                    "categorynum": "001001005",
                    "categoryname": "中标结果公示"
                },
                {
                    "categorynum": "001001006",
                    "categoryname": "项目经理变更"
                },
                {
                    "categorynum": "001001009",
                    "categoryname": "评定分离"
                }
            ],
            "zfcg": [
                {"construction_type": '政府采购'},
                {
                    "categorynum": "001002001",
                    "categoryname": "采购预告"
                },
                {
                    "categorynum": "001002002",
                    "categoryname": "采购公告"
                },
                {
                    "categorynum": "001002003",
                    "categoryname": "变更公告"
                },
                {
                    "categorynum": "001002004",
                    "categoryname": "成交公告"
                },
                {
                    "categorynum": "001002005",
                    "categoryname": "采购方式公示"
                },
                {
                    "categorynum": "001002006",
                    "categoryname": "合同公示"
                }
            ],
            "gycq": [
                {"construction_type": '国有产权'},
                {
                    "categorynum": "001004001",
                    "categoryname": "转让公告"
                },
                {
                    "categorynum": "001004002",
                    "categoryname": "司法拍卖公告"
                },
                {
                    "categorynum": "001004003",
                    "categoryname": "成交公告"
                }
            ],
            "tdykeq": [
                {"construction_type": '土地与矿业权'},
                {
                    "categorynum": "001003001",
                    "categoryname": "出让公告"
                },
                {
                    "categorynum": "001003002",
                    "categoryname": "出让土地实时报价"
                },
                {
                    "categorynum": "001003003",
                    "categoryname": "成交结果公示"
                }
            ],
            "zhjy": [
                {"construction_type": '综合交易'},
                {
                    "categorynum": "001005001",
                    "categoryname": "交易公告"
                },
                {
                    "categorynum": "001005002",
                    "categoryname": "成交公告"
                },
                {
                    "categorynum": "001005003",
                    "categoryname": "涉军交易公告"
                },
                {
                    "categorynum": "001005004",
                    "categoryname": "涉军交易公示"
                }
            ],
            "slgc": [
                {"construction_type": '水利工程'},
                {
                    "categorynum": "001006001",
                    "categoryname": "招标公告"
                },
                {
                    "categorynum": "001006003",
                    "categoryname": "中标候选人公示"
                },
                {
                    "categorynum": "001006002",
                    "categoryname": "中标人公告"
                }

            ]
        }
        return classify

    def start_requests(self):
        project_area_list = ['镇江市区', '丹阳市', '句容市', '扬中市', '丹徒区', '镇江新区']
        classify = self.classify()
        url = "http://ggzy.zhenjiang.gov.cn/inteligentsearch/rest/inteligentSearch/getFullTextData"

        for project_area in project_area_list:
            for i in classify:
                classify_second = classify[i]
                construction_type = classify_second[0]
                for j in range(1, len(classify_second)):
                    payloaddata_categorynum = classify_second[j]['categorynum']
                    item_categoryname = classify_second[j]['categoryname']
                    body = json.dumps(self.get_payloaddata(0, project_area, payloaddata_categorynum))
                    yield scrapy.FormRequest(url=url,
                                             headers=self.get_headers(),
                                             callback=self.handle_response,
                                             body=body,
                                             meta={
                                                 'url': "http://ggzy.zhenjiang.gov.cn/inteligentsearch/rest/inteligentSearch/getFullTextData",
                                                 'needbreak': False,
                                                 'construction_type': construction_type,
                                                 'payloaddata_categorynum': payloaddata_categorynum,
                                                 'item_categoryname': item_categoryname,
                                                 'project_area': project_area,
                                                 'page_pn': 0
                                             },
                                             dont_filter=True
                                             )

    def handle_response(self, response):
        json_obj = response.json()
        if not json_obj['result']['totalcount'] or json.loads(response.request.body)['pn'] >= 980:
            pass
        else:
            for row in json_obj['result']['records']:
                item = dict()
                item['release_time'] = row['infodatepx'][0:10]
                if not self.full_dose and item['release_time'] != get_current_date():
                    response.request.meta['needbreak'] = True
                item['announcement_title'] = row['title']
                html_categorynum = row['categorynum']
                url_num_s = ""
                for i in range(2, len(html_categorynum) // 3 + 1):
                    url_num_s = url_num_s + "/" + html_categorynum[0:i * 3]
                item['origin_url'] = 'http://ggzy.zhenjiang.gov.cn/jyxx' + url_num_s + '/' + ''.join(
                    item['release_time'].split('-')) + '/' + row['infoid'][0] + '.html'
                item['construction_type'] = response.request.meta['construction_type']
                item['project_area'] = response.request.meta['project_area']
                item['info_type'] = response.request.meta['item_categoryname']

                yield scrapy.Request(url=item['origin_url'], callback=self.parse_html, meta=item)

            if not response.request.meta['needbreak']:
                page_pn = response.request.meta['page_pn'] + 20
                url = response.request.meta['url']
                construction_type = response.request.meta['construction_type']
                payloaddata_categorynum = response.request.meta['payloaddata_categorynum']
                item_categoryname = response.request.meta['item_categoryname']
                project_area = response.request.meta['project_area']
                body = json.dumps(self.get_payloaddata(page_pn, project_area, payloaddata_categorynum))
                yield scrapy.FormRequest(url=url,
                                         headers=self.get_headers(),
                                         callback=self.handle_response,
                                         body=body,
                                         meta={
                                             'needbreak': False,
                                             'construction_type': construction_type,
                                             'payloaddata_categorynum': payloaddata_categorynum,
                                             'item_categoryname': item_categoryname,
                                             'project_area': project_area,
                                             'page_pn': page_pn
                                         },
                                         dont_filter=True
                                         )

    def get_headers(self):
        headers = {
            "Host": "ggzy.zhenjiang.gov.cn",
            "Connection": "keep-alive",
            # "Content-Length": "427",
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "X-Requested-With": "XMLHttpRequest",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36",
            "Content-Type": "application/json;charset=UTF-8",
            "Origin": "http://ggzy.zhenjiang.gov.cn",
            "Referer": "http://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
        }
        return headers

    def get_payloaddata(self, start_num, project_area, payloaddata_categorynum):
        payloadData = {
            "token": "",
            "pn": start_num,
            "rn": 20,
            "sdt": "",
            "edt": "",
            "wd": 'null',
            "inc_wd": "",
            "exc_wd": "",
            "fields": "title",
            "cnum": "001",
            "sort": '{"infodatepx":"0"}',
            "ssort": "title",
            "cl": 200,
            "terminal": "",
            "condition": [
                {
                    "fieldName": "zhuanzai",
                    "isLike": 'true',
                    "likeType": 2,
                    "equal": f"{project_area}"
                },
                {
                    "fieldName": "categorynum",
                    "isLike": 'true',
                    "likeType": 2,
                    "equal": f"{payloaddata_categorynum}"
                }
            ],
            "time": [
                {
                    "fieldName": "infodatepx",
                    "startTime": "2000-01-01 00:00:00",  # 起始时间
                    "endTime": f"{datetime.now().strftime('%Y-%m-%d ' + '23:59:59')}"  # 当天时间
                }
            ],
            "highlights": "title",
            "statistics": 'null',
            "unionCondition": 'null',
            "accuracy": "",
            "noParticiple": "1",
            "searchRange": 'null',
            "isBusiness": "1"
        }
        return payloadData

    def parse_html(self, response):
        item = ZhenJiangGongGongZiYuanJiaoYiZhongXinItem()
        item['html'] = response.text#[0:40]
        item["announcement_title"] = response.meta['announcement_title']
        item["release_time"] = response.meta["release_time"]
        item['project_area'] = response.meta['project_area']
        item['origin_url'] = response.meta['origin_url']
        item["construction_type"] = response.meta['construction_type']
        item['info_type'] = response.meta['info_type']
        item['is_parsed'] = 0
        item['source_type'] = self.name_zh
        item['province'] = self.province
        item['city'] = self.city
        yield item
